Getting all up in Brent’s biz - Candidate Models
Samples
library(modelr)
library(recipes)Basic samples
Perform a basic 70:30 split
set.seed(20180409)
brent_dt %>%
modelr::resample_partition(c(train=0.7,test=0.3)) ->
splits
splits %>%
pluck("train") %>%
as_tibble()->
train_raw
splits %>%
pluck("test") %>%
as_tibble()->
test_rawStart our recipe for processing our data
train_raw %>%
recipe( .) %>%
add_role(any_purchase, new_role = "outcome") %>%
add_role(email_user_id, new_role = "id") %>%
add_role(everything(),-any_purchase, -email_user_id, new_role = "predictor")->
starter_recipe
starter_recipeData Recipe
Inputs:
role #variables
id 1
outcome 1
predictor 188Perform some steps to remove columns and fill in missing country
starter_recipe %>%
step_rm(has_role("id")) %>%
step_corr(all_numeric()) %>%
step_bagimpute(country_code, seed_val = 42) %>%
step_other(country_code) %>%
step_zv(all_predictors()) %>%
prep(train_raw) ->
filter_recipe
filter_recipeData Recipe
Inputs:
role #variables
id 1
outcome 1
predictor 188
Training data contained 69362 data points and 13773 incomplete rows.
Operations:
Variables removed email_user_id [trained]
Correlation filter removed h10_clicks, h11_clicks, h6_clicks, h8_clicks, ... [trained]
Bagged tree imputation for country_code [trained]
Collapsing factor levels for country_code [trained]
Zero variance filter removed no terms [trained]train_b <- bake(filter_recipe, newdata = train_raw)
test_b <- bake(filter_recipe, newdata = test_raw)
train_bPerform numeric standardisation steps.
train_b %>%
recipe(any_purchase~.) %>%
step_center(all_numeric()) %>%
step_scale(all_numeric()) %>%
prep(train_b) ->
standardise_recipe
standardise_recipeData Recipe
Inputs:
role #variables
outcome 1
predictor 135
Training data contained 69362 data points and no missing data.
Operations:
Centering for member_rating, holiday_open_prop, ... [trained]
Scaling for member_rating, holiday_open_prop, ... [trained]train_std <- bake(standardise_recipe, train_b)
test_std <- bake(standardise_recipe, test_b)
train_stdOversample training purchases
set.seed(20180101)
filter_recipe %>%
step_upsample(all_outcomes(), ratio= .25) %>%
prep(retain=TRUE) %>%
juice() %>%
# hack because juice isn't reducing the column set
bake(filter_recipe, .) ->
train_up
print(paste("New rows:", nrow(train_up)- nrow(train_b)))[1] "New rows: 15726"Now standardise it
train_ups <- bake(standardise_recipe, train_up) Synthesised training purchases
library(synthpop)
brent_dt %>%
filter(any_purchase=="TRUE") %>%
bake(filter_recipe,.) %>%
syn(k=30000) ->
synth_purchasesSample(s) of size 30000 will be generated from original data of size 1836.
Variable any_purchase has only one value so its method has been changed to "constant".
Variable any_purchase removed as predictor because only one value.
syn variables
1 member_rating country_code any_purchase X2016.04.Early.Access.Coupon X6.Month.DBA.Training.Plan Blog.Posts DBAreactions.Tuesdays...Fridays First.Responder.Kit.Updates Monday.Links...our.favorite.SQL...tech.news.from.the.week SQL.Server.Updates
Unknown.recd X2017.05.Intersection.Orlando.Pre.Con X2017.05.SQLDay.Poland.Pre.Con X6.Month.DBA.Plan Clients Contact.Form Dell.DBA.Days.2016 EBook.Setup FeedBurner First.Responder.Kit
iTunes.Rating MailOrderDBAs.com Members OptinMonster...BrentOzar.com Sitewide.Footer SQLServerUpdates.com Unknown.Signup.Source..Before.2014.Sept. Unknown.source Watch.Brent.Tune.Queries Webcast.Drawing
Webcast.Registration holiday_open_prop weekday_open_raw weekday_open_prop weekend_open_raw weekend_open_prop h0_opens h1_opens h10_opens h11_opens
h12_opens h13_opens h14_opens h2_opens h3_opens h4_opens h5_opens h6_opens h7_opens h8_opens
h9_opens h0_opensactive h1_opensactive h10_opensactive h11_opensactive h12_opensactive h13_opensactive h14_opensactive h2_opensactive h3_opensactive
h4_opensactive h5_opensactive h6_opensactive h7_opensactive h8_opensactive h9_opensactive h0_opensprop h1_opensprop h10_opensprop h11_opensprop
h12_opensprop h13_opensprop h14_opensprop h2_opensprop h3_opensprop h4_opensprop h5_opensprop h6_opensprop h7_opensprop h8_opensprop
h9_opensprop holiday_clicks_prop weekday_clicks_prop weekend_clicks_raw weekend_clicks_prop h0_clicks h1_clicks h12_clicks h13_clicks h14_clicks
h2_clicks h3_clicks h4_clicks h7_clicks h0_clickactive h1_clickactive h10_clickactive h11_clickactive h12_clickactive h13_clickactive
h14_clickactive h2_clickactive h3_clickactive h4_clickactive h5_clickactive h6_clickactive h7_clickactive h8_clickactive h9_clickactive h0_clickprop
h1_clickprop h10_clickprop h11_clickprop h12_clickprop h13_clickprop h14_clickprop h2_clickprop h3_clickprop h4_clickprop h5_clickprop
h6_clickprop h7_clickprop h8_clickprop h9_clickprop topic_8_B_clicks topic_NA_B_clicks topic_NA_nB_clicks topic_8_B_prop topic_8_nB_prop topic_NA_B_prop
topic_NA_nB_prop topic_8_B_active topic_NA_B_active topic_NA_nB_active never_opened never_clickedsynth_purchases %>%
pluck("syn") %>%
union(train_b) %>%
sample_n(nrow(.)) ->
train_syntrain_syn_std<- bake(standardise_recipe,train_syn) Preparing
We’ll want to build models against our training sets so let’s save some code by making a big list we can map models to.
train_sets<-list(
"Basic"=train_b,
"Standardised"=train_std,
"Upsampled"=train_up,
"Upsampled & Standardised"=train_ups,
"Synthesised"=train_syn,
"Synthesised & Standardised" =train_syn_std
)Let’s remove a high RAM object.
rm("filter_recipe")Models - Basic glms
library(broom)
library(ggplot2)
train_sets %>%
map(~glm(any_purchase~ member_rating+ First.Responder.Kit+ Unknown.recd+ Monday.Links...our.favorite.SQL...tech.news.from.the.week+ weekday_open_prop + weekday_clicks_prop,
data=.,
family="binomial",
y=FALSE,x=FALSE,model=FALSE)) ->
basic_glm
saveRDS(basic_glm, "../outputs/glms.rds")basic_glm %>%
map_df(tidy, .id = "set") %>%
filter(term!="(Intercept)") %>%
ggplot(aes(x=term, y=estimate, colour=set)) +
geom_point(alpha=.5, size=3) +
coord_flip() +
ggthemes::theme_fivethirtyeight()+
geom_hline(aes(yintercept=0), colour="darkgrey", linetype="dashed")
Attributes with values consistently above 0 (dashed line) increase likelihood to buy training, and those below reduce the likelihood. The further away from the 0, the greater the impact.
library(optiRum)
vals_to_conv<-seq(-1.5,1,by=0.5)
data_frame(Coefficient=vals_to_conv,
`Odds Ratio (p/p-1)`= round(logit.odd(vals_to_conv),2),
`Probability` = round(logit.prob(vals_to_conv),2))Models - Basic tree models
library(FFTrees)
train_sets %>%
map(~dplyr::select(mutate(., any_purchase=as.logical(any_purchase)),
member_rating:weekend_open_prop,
holiday_clicks_prop:weekend_clicks_prop)) %>%
map(~FFTrees(any_purchase~., .,
goal="bacc",do.comp = FALSE,progress = FALSE)) ->
basic_trees
saveRDS(basic_trees, "../outputs/trees.rds")basic_trees %>%
map_df(~cbind(.$tree.definitions,.$tree.stats$train),.id = "set") ->
basic_tree_results
basic_tree_resultsThe chart shows different trees built using the different data sets and the balanced accuracy measure. Balanced accuracy is average of the proportion of purchases correctly classified and the proportion of non-purchases correctly classified.
basic_tree_results %>%
ggplot(aes(x=cues, y=bacc, colour=set)) +
geom_point(alpha=.5, size=3) +
coord_flip() +
ggthemes::theme_fivethirtyeight()
We can look at the difference between the correctly classified purchases (red) and the correctly classified non-purchases (grey) for the trees with the highest balanced accuracy. There’s two visible cases:
- high accuracy for purchases and low accuracy for non-purchases
- OK accuracy for purchases and good accuracy for non-purchases
Selection one of these would depend on the cost of the low accuracy for non-purchases.
basic_tree_results %>%
top_n(10, bacc) %>%
mutate(cues=stringr::str_wrap(stringr::str_replace_all(cues,";", " "),1)) %>%
dplyr::select(cues, sens, spec, set) %>%
ggplot(aes(x=set, y=spec, ymax=sens, ymin=spec)) +
geom_pointrange(alpha=0.3, size=2)+
geom_point(aes(y=sens), colour="red", size=6)+
facet_wrap(~cues, ncol=4)+
ggthemes::theme_fivethirtyeight()
Models - glmnets
glmnets take all our variables and construct a model with them. However, a model with 135 coefficients would be pretty crazy. Instead glmnet downweights coefficients towards 0 as it penalises complexity. As such, even though our model notionally contains all our columns not everything will contribute.
library(glmnet)
library(glmnetUtils)
train_sets %>%
map(~cv.glmnet(any_purchase~., data=. ,
family = "binomial",type.measure="class",
standardize = FALSE)) ->
basic_glmnets
saveRDS(basic_glmnets, "../outputs/glmnets.rds")basic_glmnets %>%
map_df(~rownames_to_column(as.data.frame(as.matrix(coefficients(.))),"col"),.id = "set" ) %>%
rename(coef=`1`) ->
basic_glmnet_coefs
basic_glmnet_coefs %>%
filter(coef!=0.000000e+00) %>%
filter(col!="(Intercept)") %>%
arrange(col) %>%
count(col) %>%
mutate( pane = row_number() %/% 38) ->
panes
basic_glmnet_coefs %>%
inner_join(panes) %>%
arrange(desc(col)) %>%
mutate(col=fct_inorder(col)) %>%
ggplot(aes(x=col, y=coef, colour=set)) +
geom_point() +
coord_flip()+
facet_wrap(~pane,scales = "free", ncol=5)+
ggthemes::theme_fivethirtyeight()
The non-standardised types of samples do not work well with glmnet and typically it would have scaled them. We can see which columns (primarily in the standardised samples) make it our models most commonly. These will be the most predictive columns overall.
panes %>%
arrange(desc(n)) %>%
mutate(col=fct_inorder(col)) %>%
mutate( pane = row_number() %/% 38) %>%
arrange(desc(col)) %>%
mutate(col=fct_inorder(col)) %>%
ggplot(aes(x=col, y=n)) +
geom_col() +
coord_flip()+
facet_wrap(~pane, ncol=5, scales = "free_y")+
ggthemes::theme_fivethirtyeight()
Models - Complex models via h2o
library(h2o)
options("h2o.use.data.table"=TRUE)
h2o.init(max_mem_size = "8G")
H2O is not running yet, starting it now...
Note: In case of errors look at the following log files:
C:\Users\steph\AppData\Local\Temp\RtmpCynpf4/h2o_steph_started_from_r.out
C:\Users\steph\AppData\Local\Temp\RtmpCynpf4/h2o_steph_started_from_r.err
java version "1.8.0_161"
Java(TM) SE Runtime Environment (build 1.8.0_161-b12)
Java HotSpot(TM) 64-Bit Server VM (build 25.161-b12, mixed mode)
Starting H2O JVM and connecting: . Connection successful!
R is connected to the H2O cluster:
H2O cluster uptime: 2 seconds 157 milliseconds
H2O cluster version: 3.16.0.2
H2O cluster version age: 4 months and 12 days !!!
H2O cluster name: H2O_started_from_R_steph_xwr366
H2O cluster total nodes: 1
H2O cluster total memory: 7.11 GB
H2O cluster total cores: 8
H2O cluster allowed cores: 8
H2O cluster healthy: TRUE
H2O Connection ip: localhost
H2O Connection port: 54321
H2O Connection proxy: NA
H2O Internal Security: FALSE
H2O API Extensions: Algos, AutoML, Core V3, Core V4
R Version: R version 3.4.4 (2018-03-15) train_sets %>%
map(as.h2o) %>%
map(~h2o.automl(y = "any_purchase", training_frame = ., seed = 1313, max_runtime_secs = 180)) ->
basic_automl
|
| | 0%
|
|=========================================================================================| 100%
|
| | 0%
|
|=========================================================================================| 100%
|
| | 0%
|
|=========================================================================================| 100%
|
| | 0%
|
|=========================================================================================| 100%
|
| | 0%
|
|=========================================================================================| 100%
|
| | 0%
|
|=========================================================================================| 100%
|
| | 0%
|
|=== | 3%
|
|==== | 5%
|
|===== | 5%
|
|===== | 6%
|
|====== | 6%
|
|====== | 7%
|
|======= | 7%
|
|======= | 8%
|
|======== | 8%
|
|======== | 9%
|
|======== | 10%
|
|========= | 10%
|
|========== | 11%
|
|========== | 12%
|
|=========== | 12%
|
|=========== | 13%
|
|============ | 13%
|
|============ | 14%
|
|============= | 14%
|
|============= | 15%
|
|============== | 15%
|
|============== | 16%
|
|=============== | 16%
|
|=============== | 17%
|
|================ | 18%
|
|================= | 19%
|
|================= | 20%
|
|================== | 20%
|
|================== | 21%
|
|=================== | 21%
|
|=================== | 22%
|
|==================== | 22%
|
|==================== | 23%
|
|===================== | 23%
|
|===================== | 24%
|
|=========================================================================================| 100%
|
| | 0%
|
|=========================================================================================| 100%
|
| | 0%
|
|=== | 3%
|
|===== | 5%
|
|===== | 6%
|
|====== | 6%
|
|====== | 7%
|
|======= | 7%
|
|======= | 8%
|
|======== | 9%
|
|========= | 10%
|
|========= | 11%
|
|========== | 11%
|
|========== | 12%
|
|=========== | 12%
|
|=========== | 13%
|
|============ | 13%
|
|============ | 14%
|
|============= | 14%
|
|============= | 15%
|
|============== | 15%
|
|============== | 16%
|
|=============== | 16%
|
|=============== | 17%
|
|================ | 18%
|
|================= | 19%
|
|================= | 20%
|
|================== | 20%
|
|================== | 21%
|
|=================== | 21%
|
|=================== | 22%
|
|==================== | 22%
|
|==================== | 23%
|
|===================== | 23%
|
|===================== | 24%
|
|=========================================================================================| 100%
|
| | 0%
|
|=========================================================================================| 100%
|
| | 0%
|
|=== | 3%
|
|==== | 5%
|
|===== | 5%
|
|===== | 6%
|
|====== | 7%
|
|======= | 8%
|
|======== | 8%
|
|======== | 9%
|
|========= | 10%
|
|========== | 11%
|
|========== | 12%
|
|=========== | 12%
|
|=========== | 13%
|
|============ | 13%
|
|============ | 14%
|
|============= | 14%
|
|============= | 15%
|
|============== | 15%
|
|============== | 16%
|
|=============== | 16%
|
|=============== | 17%
|
|================ | 18%
|
|================= | 19%
|
|================= | 20%
|
|================== | 20%
|
|================== | 21%
|
|=================== | 21%
|
|=========================================================================================| 100%
|
| | 0%
|
|=========================================================================================| 100%
|
| | 0%
|
|=== | 3%
|
|==== | 5%
|
|===== | 5%
|
|===== | 6%
|
|====== | 7%
|
|======= | 8%
|
|======== | 8%
|
|======== | 9%
|
|========= | 10%
|
|========== | 11%
|
|========== | 12%
|
|=========== | 12%
|
|=========== | 13%
|
|============ | 13%
|
|============ | 14%
|
|============= | 14%
|
|============= | 15%
|
|============== | 15%
|
|============== | 16%
|
|=============== | 16%
|
|=============== | 17%
|
|================ | 18%
|
|================= | 19%
|
|================= | 20%
|
|================== | 20%
|
|================== | 21%
|
|=================== | 21%
|
|=========================================================================================| 100%
|
| | 0%
|
|=========================================================================================| 100%
|
| | 0%
|
|=== | 3%
|
|==== | 5%
|
|===== | 5%
|
|===== | 6%
|
|====== | 6%
|
|====== | 7%
|
|======= | 7%
|
|======= | 8%
|
|======== | 8%
|
|======== | 9%
|
|======== | 10%
|
|========= | 10%
|
|========= | 11%
|
|========== | 11%
|
|========== | 12%
|
|=========== | 12%
|
|=========== | 13%
|
|============ | 13%
|
|============ | 14%
|
|============= | 14%
|
|============= | 15%
|
|============== | 15%
|
|============== | 16%
|
|=============== | 16%
|
|=============== | 17%
|
|================ | 18%
|
|================= | 19%
|
|================= | 20%
|
|================== | 20%
|
|===================================================================================== | 95%
|
|=========================================================================================| 100%
|
| | 0%
|
|=========================================================================================| 100%
|
| | 0%
|
|=== | 3%
|
|===== | 5%
|
|===== | 6%
|
|====== | 6%
|
|====== | 7%
|
|======= | 7%
|
|======= | 8%
|
|======== | 8%
|
|======== | 9%
|
|======== | 10%
|
|========= | 10%
|
|========= | 11%
|
|========== | 11%
|
|========== | 12%
|
|=========== | 12%
|
|=========== | 13%
|
|============ | 13%
|
|============ | 14%
|
|============= | 14%
|
|============= | 15%
|
|============== | 15%
|
|============== | 16%
|
|=============== | 16%
|
|=============== | 17%
|
|================ | 18%
|
|================= | 19%
|
|================= | 20%
|
|================== | 20%
|
|===================================================================================== | 95%
|
|=========================================================================================| 100%
|
| | 0%
|
|=========================================================================================| 100%saveRDS(basic_automl, "../outputs/automl.rds")Evaluation
LS0tDQp0aXRsZTogIkdldHRpbmcgYWxsIHVwIGluIEJyZW50J3MgYml6IC0gQ2FuZGlkYXRlIE1vZGVscyINCm91dHB1dDogDQogIGh0bWxfbm90ZWJvb2s6IA0KICAgIGNvZGVfZm9sZGluZzogaGlkZQ0KICAgIHRvYzogeWVzDQogICAgdG9jX2RlcHRoOiAxDQotLS0NCg0KYGBge3Igc2V0dXAsIGluY2x1ZGU9RkFMU0V9DQprbml0cjo6b3B0c19jaHVuayRzZXQod2FybmluZyA9IEZBTFNFLCBtZXNzYWdlID0gRkFMU0UpDQpsaWJyYXJ5KHRpZHl2ZXJzZSkNCmxpYnJhcnkoZHRwbHlyKQ0KYnJlbnRfZHQ8LWRhdGEudGFibGU6OmZyZWFkKCIuLi9vdXRwdXRzL2ZpbmFsZGF0YXNldC5jc3YiKQ0KYnJlbnRfZHRbLCBwdXJjaGFzZTo9TlVMTF0NCmJyZW50X2R0ICU+JSANCiAgYXNfdGliYmxlKCkgJT4lIA0KICBzZXRfdGlkeV9uYW1lcyhzeW50YWN0aWMgPSBUUlVFKSAlPiUgDQogIG11dGF0ZV9pZihpc19sb2dpY2FsLCBmYWN0b3IsIGxldmVscz1jKCJGQUxTRSIsIlRSVUUiKSkgJT4lIA0KICBtdXRhdGVfaWYoaXNfY2hhcmFjdGVyLCBhc19mYWN0b3IpICU+JSANCiAgbXV0YXRlX2lmKH5pcy5mYWN0b3IoLikmbl9kaXN0aW5jdCguKSE9MiwgZmN0X2luZnJlcSkgLT4NCiAgYnJlbnRfZHQNCg0KZG9fZ2xtPC0hZmlsZS5leGlzdHMoIi4uL291dHB1dHMvZ2xtcy5yZHMiKQ0KZG9fZ2xtbmV0czwtIWZpbGUuZXhpc3RzKCIuLi9vdXRwdXRzL2dsbW5ldHMucmRzIikNCmRvX3RyZWVzPC0hZmlsZS5leGlzdHMoIi4uL291dHB1dHMvdHJlZXMucmRzIikNCmRvX2F1dG9tbDwtIWZpbGUuZXhpc3RzKCIuLi9vdXRwdXRzL2F1dG90bWwucmRzIikNCmBgYA0KDQojIFNhbXBsZXMNCmBgYHtyfQ0KbGlicmFyeShtb2RlbHIpDQpsaWJyYXJ5KHJlY2lwZXMpDQpgYGANCg0KIyMgQmFzaWMgc2FtcGxlcw0KDQpQZXJmb3JtIGEgYmFzaWMgNzA6MzAgc3BsaXQNCg0KYGBge3J9DQpzZXQuc2VlZCgyMDE4MDQwOSkNCmJyZW50X2R0ICU+JSANCiAgbW9kZWxyOjpyZXNhbXBsZV9wYXJ0aXRpb24oYyh0cmFpbj0wLjcsdGVzdD0wLjMpKSAtPg0KICBzcGxpdHMNCg0Kc3BsaXRzICU+JSANCiAgcGx1Y2soInRyYWluIikgJT4lIA0KICBhc190aWJibGUoKS0+DQogIHRyYWluX3Jhdw0KDQpzcGxpdHMgJT4lIA0KICBwbHVjaygidGVzdCIpICU+JSANCiAgYXNfdGliYmxlKCktPg0KICB0ZXN0X3Jhdw0KYGBgDQoNClN0YXJ0IG91ciByZWNpcGUgZm9yIHByb2Nlc3Npbmcgb3VyIGRhdGENCg0KYGBge3J9DQp0cmFpbl9yYXcgJT4lIA0KICByZWNpcGUoIC4pICAlPiUNCiAgYWRkX3JvbGUoYW55X3B1cmNoYXNlLCBuZXdfcm9sZSA9ICJvdXRjb21lIikgJT4lIA0KICBhZGRfcm9sZShlbWFpbF91c2VyX2lkLCBuZXdfcm9sZSA9ICJpZCIpICU+JSANCiAgYWRkX3JvbGUoZXZlcnl0aGluZygpLC1hbnlfcHVyY2hhc2UsIC1lbWFpbF91c2VyX2lkLCBuZXdfcm9sZSA9ICJwcmVkaWN0b3IiKS0+DQogIHN0YXJ0ZXJfcmVjaXBlDQoNCnN0YXJ0ZXJfcmVjaXBlDQpgYGANCg0KUGVyZm9ybSBzb21lIHN0ZXBzIHRvIHJlbW92ZSBjb2x1bW5zIGFuZCBmaWxsIGluIG1pc3NpbmcgY291bnRyeQ0KYGBge3J9DQpzdGFydGVyX3JlY2lwZSAlPiUgDQogIHN0ZXBfcm0oaGFzX3JvbGUoImlkIikpICU+JSANCiAgc3RlcF9jb3JyKGFsbF9udW1lcmljKCkpICAlPiUgDQogIHN0ZXBfYmFnaW1wdXRlKGNvdW50cnlfY29kZSwgc2VlZF92YWwgPSA0MikgJT4lIA0KICBzdGVwX290aGVyKGNvdW50cnlfY29kZSkgJT4lICANCiAgc3RlcF96dihhbGxfcHJlZGljdG9ycygpKSAgJT4lIA0KICBwcmVwKHRyYWluX3JhdykgLT4NCiAgZmlsdGVyX3JlY2lwZQ0KDQpmaWx0ZXJfcmVjaXBlDQpgYGANCg0KYGBge3J9DQp0cmFpbl9iIDwtIGJha2UoZmlsdGVyX3JlY2lwZSwgbmV3ZGF0YSA9IHRyYWluX3JhdykgDQp0ZXN0X2IgIDwtIGJha2UoZmlsdGVyX3JlY2lwZSwgbmV3ZGF0YSA9IHRlc3RfcmF3KSANCg0KdHJhaW5fYg0KYGBgDQoNCg0KDQpQZXJmb3JtIG51bWVyaWMgc3RhbmRhcmRpc2F0aW9uIHN0ZXBzLg0KDQpgYGB7cn0NCnRyYWluX2IgJT4lIA0KICByZWNpcGUoYW55X3B1cmNoYXNlfi4pICU+JSANCiAgc3RlcF9jZW50ZXIoYWxsX251bWVyaWMoKSkgJT4lIA0KICBzdGVwX3NjYWxlKGFsbF9udW1lcmljKCkpICU+JSANCiAgcHJlcCh0cmFpbl9iKSAtPg0KICBzdGFuZGFyZGlzZV9yZWNpcGUNCg0Kc3RhbmRhcmRpc2VfcmVjaXBlDQpgYGANCg0KDQpgYGB7cn0NCnRyYWluX3N0ZCA8LSBiYWtlKHN0YW5kYXJkaXNlX3JlY2lwZSwgdHJhaW5fYikgDQp0ZXN0X3N0ZCAgPC0gYmFrZShzdGFuZGFyZGlzZV9yZWNpcGUsIHRlc3RfYikgDQoNCnRyYWluX3N0ZA0KYGBgDQoNCiMjIE92ZXJzYW1wbGUgdHJhaW5pbmcgcHVyY2hhc2VzIA0KYGBge3J9DQpzZXQuc2VlZCgyMDE4MDEwMSkNCmZpbHRlcl9yZWNpcGUgJT4lIA0KICBzdGVwX3Vwc2FtcGxlKGFsbF9vdXRjb21lcygpLCByYXRpbz0gLjI1KSAlPiUgDQogIHByZXAocmV0YWluPVRSVUUpICU+JSANCiAganVpY2UoKSAlPiUgDQogICMgaGFjayBiZWNhdXNlIGp1aWNlIGlzbid0IHJlZHVjaW5nIHRoZSBjb2x1bW4gc2V0DQogIGJha2UoZmlsdGVyX3JlY2lwZSwgLikgLT4NCiAgdHJhaW5fdXANCg0KcHJpbnQocGFzdGUoIk5ldyByb3dzOiIsIG5yb3codHJhaW5fdXApLSBucm93KHRyYWluX2IpKSkNCmBgYA0KDQpOb3cgc3RhbmRhcmRpc2UgaXQNCmBgYHtyfQ0KdHJhaW5fdXBzIDwtIGJha2Uoc3RhbmRhcmRpc2VfcmVjaXBlLCB0cmFpbl91cCkgDQpgYGANCg0KIyMgU3ludGhlc2lzZWQgdHJhaW5pbmcgcHVyY2hhc2VzIA0KYGBge3J9DQpsaWJyYXJ5KHN5bnRocG9wKQ0KYnJlbnRfZHQgJT4lIA0KICBmaWx0ZXIoYW55X3B1cmNoYXNlPT0iVFJVRSIpICU+JSANCiAgYmFrZShmaWx0ZXJfcmVjaXBlLC4pICU+JSANCiAgc3luKGs9MzAwMDApICAtPg0KICBzeW50aF9wdXJjaGFzZXMNCg0Kc3ludGhfcHVyY2hhc2VzICU+JSANCiAgcGx1Y2soInN5biIpICU+JSANCiAgdW5pb24odHJhaW5fYikgJT4lIA0KICBzYW1wbGVfbihucm93KC4pKSAtPg0KICB0cmFpbl9zeW4NCmBgYA0KDQpgYGB7cn0NCnRyYWluX3N5bl9zdGQ8LSBiYWtlKHN0YW5kYXJkaXNlX3JlY2lwZSx0cmFpbl9zeW4pIA0KYGBgDQoNCiMjIFByZXBhcmluZw0KV2UnbGwgd2FudCB0byBidWlsZCBtb2RlbHMgYWdhaW5zdCBvdXIgdHJhaW5pbmcgc2V0cyBzbyBsZXQncyBzYXZlIHNvbWUgY29kZSBieSBtYWtpbmcgYSBiaWcgbGlzdCB3ZSBjYW4gbWFwIG1vZGVscyB0by4NCg0KYGBge3J9DQp0cmFpbl9zZXRzPC1saXN0KA0KICAiQmFzaWMiPXRyYWluX2IsDQogICJTdGFuZGFyZGlzZWQiPXRyYWluX3N0ZCwNCiAgIlVwc2FtcGxlZCI9dHJhaW5fdXAsDQogICJVcHNhbXBsZWQgJiBTdGFuZGFyZGlzZWQiPXRyYWluX3VwcywNCiAgIlN5bnRoZXNpc2VkIj10cmFpbl9zeW4sDQogICJTeW50aGVzaXNlZCAmIFN0YW5kYXJkaXNlZCIgPXRyYWluX3N5bl9zdGQNCikNCmBgYA0KDQpMZXQncyByZW1vdmUgYSBoaWdoIFJBTSBvYmplY3QuDQoNCmBgYHtyfQ0Kcm0oImZpbHRlcl9yZWNpcGUiKQ0KYGBgDQoNCiMgTW9kZWxzIC0gQmFzaWMgZ2xtcw0KYGBge3IgIGV2YWw9ZG9fZ2xtLCBlY2hvPVRSVUV9DQpsaWJyYXJ5KGJyb29tKQ0KbGlicmFyeShnZ3Bsb3QyKQ0KdHJhaW5fc2V0cyAlPiUgDQogIG1hcCh+Z2xtKGFueV9wdXJjaGFzZX4gIG1lbWJlcl9yYXRpbmcrIEZpcnN0LlJlc3BvbmRlci5LaXQrIFVua25vd24ucmVjZCsgTW9uZGF5LkxpbmtzLi4ub3VyLmZhdm9yaXRlLlNRTC4uLnRlY2gubmV3cy5mcm9tLnRoZS53ZWVrKyB3ZWVrZGF5X29wZW5fcHJvcCArIHdlZWtkYXlfY2xpY2tzX3Byb3AsIA0KICAgICAgICAgICBkYXRhPS4sDQogICAgICAgICAgIGZhbWlseT0iYmlub21pYWwiLA0KICAgICAgICAgICB5PUZBTFNFLHg9RkFMU0UsbW9kZWw9RkFMU0UpKSAgLT4NCiAgYmFzaWNfZ2xtDQoNCnNhdmVSRFMoYmFzaWNfZ2xtLCAiLi4vb3V0cHV0cy9nbG1zLnJkcyIpDQpgYGANCg0KYGBge3IgZXZhbD0hZG9fZ2xtLCBlY2hvPUZBTFNFfQ0KYmFzaWNfZ2xtPC1yZWFkX3JkcygiLi4vb3V0cHV0cy9nbG1zLnJkcyIpDQpgYGANCg0KYGBge3IgZmlnLndpZHRoPTEwfQ0KYmFzaWNfZ2xtICU+JSANCiAgbWFwX2RmKHRpZHksIC5pZCA9ICJzZXQiKSAlPiUgDQogIGZpbHRlcih0ZXJtIT0iKEludGVyY2VwdCkiKSAlPiUgDQogIGdncGxvdChhZXMoeD10ZXJtLCB5PWVzdGltYXRlLCBjb2xvdXI9c2V0KSkgKw0KICBnZW9tX3BvaW50KGFscGhhPS41LCBzaXplPTMpICArDQogIGNvb3JkX2ZsaXAoKSArDQogIGdndGhlbWVzOjp0aGVtZV9maXZldGhpcnR5ZWlnaHQoKSsNCiAgZ2VvbV9obGluZShhZXMoeWludGVyY2VwdD0wKSwgY29sb3VyPSJkYXJrZ3JleSIsIGxpbmV0eXBlPSJkYXNoZWQiKQ0KYGBgDQoNCkF0dHJpYnV0ZXMgd2l0aCB2YWx1ZXMgY29uc2lzdGVudGx5IGFib3ZlIDAgKGRhc2hlZCBsaW5lKSBpbmNyZWFzZSBsaWtlbGlob29kIHRvIGJ1eSB0cmFpbmluZywgYW5kIHRob3NlIGJlbG93IHJlZHVjZSB0aGUgbGlrZWxpaG9vZC4gVGhlIGZ1cnRoZXIgYXdheSBmcm9tIHRoZSAwLCB0aGUgZ3JlYXRlciB0aGUgaW1wYWN0LiANCg0KYGBge3J9DQpsaWJyYXJ5KG9wdGlSdW0pDQp2YWxzX3RvX2NvbnY8LXNlcSgtMS41LDEsYnk9MC41KQ0KZGF0YV9mcmFtZShDb2VmZmljaWVudD12YWxzX3RvX2NvbnYsDQogICAgICAgICAgIGBPZGRzIFJhdGlvIChwL3AtMSlgPSByb3VuZChsb2dpdC5vZGQodmFsc190b19jb252KSwyKSwNCiAgICAgICAgICAgYFByb2JhYmlsaXR5YCA9IHJvdW5kKGxvZ2l0LnByb2IodmFsc190b19jb252KSwyKSkNCmBgYA0KDQojIE1vZGVscyAtIEJhc2ljIHRyZWUgbW9kZWxzDQpgYGB7ciBldmFsPWRvX3RyZWVzLCBlY2hvPVRSVUV9DQpsaWJyYXJ5KEZGVHJlZXMpDQp0cmFpbl9zZXRzICU+JSANCiAgbWFwKH5kcGx5cjo6c2VsZWN0KG11dGF0ZSguLCBhbnlfcHVyY2hhc2U9YXMubG9naWNhbChhbnlfcHVyY2hhc2UpKSwNCiAgICAgICAgICAgICAgbWVtYmVyX3JhdGluZzp3ZWVrZW5kX29wZW5fcHJvcCwgDQogICAgICAgICAgICAgIGhvbGlkYXlfY2xpY2tzX3Byb3A6d2Vla2VuZF9jbGlja3NfcHJvcCkpICU+JSANCiAgbWFwKH5GRlRyZWVzKGFueV9wdXJjaGFzZX4uLCAuLA0KICAgICAgICAgICAgICAgZ29hbD0iYmFjYyIsZG8uY29tcCA9IEZBTFNFLHByb2dyZXNzID0gRkFMU0UpKSAtPg0KICBiYXNpY190cmVlcw0Kc2F2ZVJEUyhiYXNpY190cmVlcywgIi4uL291dHB1dHMvdHJlZXMucmRzIikNCmBgYA0KDQpgYGB7ciBldmFsPSFkb19nbG0sIGVjaG89RkFMU0V9DQpiYXNpY190cmVlczwtcmVhZF9yZHMoIi4uL291dHB1dHMvdHJlZXMucmRzIikNCmBgYA0KDQpgYGB7cn0NCmJhc2ljX3RyZWVzICU+JSANCiAgbWFwX2RmKH5jYmluZCguJHRyZWUuZGVmaW5pdGlvbnMsLiR0cmVlLnN0YXRzJHRyYWluKSwuaWQgPSAic2V0IikgLT4NCiAgYmFzaWNfdHJlZV9yZXN1bHRzDQoNCmJhc2ljX3RyZWVfcmVzdWx0cw0KYGBgDQoNCg0KDQpUaGUgY2hhcnQgc2hvd3MgZGlmZmVyZW50IHRyZWVzIGJ1aWx0IHVzaW5nIHRoZSBkaWZmZXJlbnQgZGF0YSBzZXRzIGFuZCB0aGUgYmFsYW5jZWQgYWNjdXJhY3kgbWVhc3VyZS4gQmFsYW5jZWQgYWNjdXJhY3kgaXMgYXZlcmFnZSBvZiB0aGUgcHJvcG9ydGlvbiBvZiBwdXJjaGFzZXMgY29ycmVjdGx5IGNsYXNzaWZpZWQgYW5kIHRoZSBwcm9wb3J0aW9uIG9mIG5vbi1wdXJjaGFzZXMgY29ycmVjdGx5IGNsYXNzaWZpZWQuDQpgYGB7ciBmaWcud2lkdGg9MTB9DQpiYXNpY190cmVlX3Jlc3VsdHMgJT4lIA0KICBnZ3Bsb3QoYWVzKHg9Y3VlcywgeT1iYWNjLCBjb2xvdXI9c2V0KSkgKw0KICBnZW9tX3BvaW50KGFscGhhPS41LCBzaXplPTMpICArDQogIGNvb3JkX2ZsaXAoKSArDQogIGdndGhlbWVzOjp0aGVtZV9maXZldGhpcnR5ZWlnaHQoKQ0KYGBgDQoNCldlIGNhbiBsb29rIGF0IHRoZSBkaWZmZXJlbmNlIGJldHdlZW4gdGhlIGNvcnJlY3RseSBjbGFzc2lmaWVkIHB1cmNoYXNlcyAocmVkKSBhbmQgdGhlIGNvcnJlY3RseSBjbGFzc2lmaWVkIG5vbi1wdXJjaGFzZXMgKGdyZXkpIGZvciB0aGUgdHJlZXMgd2l0aCB0aGUgaGlnaGVzdCBiYWxhbmNlZCBhY2N1cmFjeS4gVGhlcmUncyB0d28gdmlzaWJsZSBjYXNlczoNCg0KMS4gaGlnaCBhY2N1cmFjeSBmb3IgcHVyY2hhc2VzIGFuZCBsb3cgYWNjdXJhY3kgZm9yIG5vbi1wdXJjaGFzZXMgDQoyLiBPSyBhY2N1cmFjeSBmb3IgcHVyY2hhc2VzIGFuZCBnb29kIGFjY3VyYWN5IGZvciBub24tcHVyY2hhc2VzDQoNClNlbGVjdGlvbiBvbmUgb2YgdGhlc2Ugd291bGQgZGVwZW5kIG9uIHRoZSBjb3N0IG9mIHRoZSBsb3cgYWNjdXJhY3kgZm9yIG5vbi1wdXJjaGFzZXMuDQoNCmBgYHtyIGZpZy53aWR0aD0xMH0NCmJhc2ljX3RyZWVfcmVzdWx0cyAlPiUgDQogIHRvcF9uKDEwLCBiYWNjKSAlPiUgDQogIG11dGF0ZShjdWVzPXN0cmluZ3I6OnN0cl93cmFwKHN0cmluZ3I6OnN0cl9yZXBsYWNlX2FsbChjdWVzLCI7IiwgIiAiKSwxKSkgJT4lIA0KICBkcGx5cjo6c2VsZWN0KGN1ZXMsIHNlbnMsIHNwZWMsIHNldCkgJT4lIA0KICBnZ3Bsb3QoYWVzKHg9c2V0LCB5PXNwZWMsIHltYXg9c2VucywgeW1pbj1zcGVjKSkgKw0KICBnZW9tX3BvaW50cmFuZ2UoYWxwaGE9MC4zLCBzaXplPTIpKw0KICBnZW9tX3BvaW50KGFlcyh5PXNlbnMpLCBjb2xvdXI9InJlZCIsIHNpemU9NikrDQogIGZhY2V0X3dyYXAofmN1ZXMsIG5jb2w9NCkrDQogIGdndGhlbWVzOjp0aGVtZV9maXZldGhpcnR5ZWlnaHQoKQ0KYGBgDQoNCg0KIyBNb2RlbHMgLSBnbG1uZXRzDQoNCmdsbW5ldHMgdGFrZSBhbGwgb3VyIHZhcmlhYmxlcyBhbmQgY29uc3RydWN0IGEgbW9kZWwgd2l0aCB0aGVtLiBIb3dldmVyLCBhIG1vZGVsIHdpdGggYHIgbmNvbCh0ZXN0X2IpLTFgIGNvZWZmaWNpZW50cyB3b3VsZCBiZSBwcmV0dHkgY3JhenkuIEluc3RlYWQgYGdsbW5ldGAgZG93bndlaWdodHMgY29lZmZpY2llbnRzIHRvd2FyZHMgMCBhcyBpdCBwZW5hbGlzZXMgY29tcGxleGl0eS4gQXMgc3VjaCwgZXZlbiB0aG91Z2ggb3VyIG1vZGVsIG5vdGlvbmFsbHkgY29udGFpbnMgYWxsIG91ciBjb2x1bW5zIG5vdCBldmVyeXRoaW5nIHdpbGwgY29udHJpYnV0ZS4NCg0KDQpgYGB7ciBldmFsPWRvX2dsbW5ldHN9DQpsaWJyYXJ5KGdsbW5ldCkNCmxpYnJhcnkoZ2xtbmV0VXRpbHMpDQp0cmFpbl9zZXRzICU+JSANCiAgbWFwKH5jdi5nbG1uZXQoYW55X3B1cmNoYXNlfi4sIGRhdGE9LiAsDQogICAgICAgICAgICAgIGZhbWlseSA9ICJiaW5vbWlhbCIsdHlwZS5tZWFzdXJlPSJjbGFzcyIsDQogICAgICAgICAgICAgIHN0YW5kYXJkaXplID0gRkFMU0UpKSAtPg0KICBiYXNpY19nbG1uZXRzDQoNCnNhdmVSRFMoYmFzaWNfZ2xtbmV0cywgIi4uL291dHB1dHMvZ2xtbmV0cy5yZHMiKQ0KYGBgDQoNCmBgYHtyIGV2YWw9IWRvX2dsbW5ldHMsIGVjaG89RkFMU0V9DQpiYXNpY19nbG1uZXRzPC1yZWFkX3JkcygiLi4vb3V0cHV0cy9nbG1uZXRzLnJkcyIpDQpgYGANCg0KYGBge3IgZmlnLmhlaWdodD0xNX0NCmJhc2ljX2dsbW5ldHMgJT4lIA0KICBtYXBfZGYofnJvd25hbWVzX3RvX2NvbHVtbihhcy5kYXRhLmZyYW1lKGFzLm1hdHJpeChjb2VmZmljaWVudHMoLikpKSwiY29sIiksLmlkID0gInNldCIgKSAlPiUgDQogIHJlbmFtZShjb2VmPWAxYCkgLT4gDQogIGJhc2ljX2dsbW5ldF9jb2Vmcw0KDQpiYXNpY19nbG1uZXRfY29lZnMgJT4lIA0KICBmaWx0ZXIoY29lZiE9MC4wMDAwMDBlKzAwKSAlPiUgDQogIGZpbHRlcihjb2whPSIoSW50ZXJjZXB0KSIpICU+JSANCiAgYXJyYW5nZShjb2wpICU+JSANCiAgY291bnQoY29sKSAlPiUgDQogIG11dGF0ZSggcGFuZSA9IHJvd19udW1iZXIoKSAlLyUgMzgpIC0+DQogIHBhbmVzDQogIA0KYmFzaWNfZ2xtbmV0X2NvZWZzICU+JSANCiAgaW5uZXJfam9pbihwYW5lcykgJT4lIA0KICBhcnJhbmdlKGRlc2MoY29sKSkgJT4lIA0KICBtdXRhdGUoY29sPWZjdF9pbm9yZGVyKGNvbCkpICU+JSANCiAgZ2dwbG90KGFlcyh4PWNvbCwgeT1jb2VmLCBjb2xvdXI9c2V0KSkgKw0KICBnZW9tX3BvaW50KCkgKyANCiAgY29vcmRfZmxpcCgpKw0KICBmYWNldF93cmFwKH5wYW5lLHNjYWxlcyA9ICJmcmVlIiwgbmNvbD01KSsNCiAgZ2d0aGVtZXM6OnRoZW1lX2ZpdmV0aGlydHllaWdodCgpDQpgYGANCg0KVGhlIG5vbi1zdGFuZGFyZGlzZWQgdHlwZXMgb2Ygc2FtcGxlcyBkbyBub3Qgd29yayB3ZWxsIHdpdGggZ2xtbmV0IGFuZCB0eXBpY2FsbHkgaXQgd291bGQgaGF2ZSBzY2FsZWQgdGhlbS4gV2UgY2FuIHNlZSB3aGljaCBjb2x1bW5zIChwcmltYXJpbHkgaW4gdGhlIHN0YW5kYXJkaXNlZCBzYW1wbGVzKSBtYWtlIGl0IG91ciBtb2RlbHMgbW9zdCBjb21tb25seS4gVGhlc2Ugd2lsbCBiZSB0aGUgbW9zdCBwcmVkaWN0aXZlIGNvbHVtbnMgb3ZlcmFsbC4NCmBgYHtyIGZpZy5oZWlnaHQ9MTV9DQpwYW5lcyAlPiUgDQogIGFycmFuZ2UoZGVzYyhuKSkgJT4lIA0KICBtdXRhdGUoY29sPWZjdF9pbm9yZGVyKGNvbCkpICU+JSANCiAgbXV0YXRlKCBwYW5lID0gcm93X251bWJlcigpICUvJSAzOCkgJT4lIA0KICBhcnJhbmdlKGRlc2MoY29sKSkgJT4lIA0KICBtdXRhdGUoY29sPWZjdF9pbm9yZGVyKGNvbCkpICU+JSANCiAgZ2dwbG90KGFlcyh4PWNvbCwgeT1uKSkgKw0KICBnZW9tX2NvbCgpICsgDQogIGNvb3JkX2ZsaXAoKSsNCiAgZmFjZXRfd3JhcCh+cGFuZSwgbmNvbD01LCBzY2FsZXMgPSAiZnJlZV95IikrDQogIGdndGhlbWVzOjp0aGVtZV9maXZldGhpcnR5ZWlnaHQoKQ0KYGBgDQoNCg0KIyBNb2RlbHMgLSBDb21wbGV4IG1vZGVscyB2aWEgaDJvDQoNCmBgYHtyIGV2YWw9ZG9fYXV0b21sfQ0KbGlicmFyeShoMm8pDQpvcHRpb25zKCJoMm8udXNlLmRhdGEudGFibGUiPVRSVUUpDQpoMm8uaW5pdChtYXhfbWVtX3NpemUgPSAiOEciKQ0KdHJhaW5fc2V0cyAlPiUgDQogIG1hcChhcy5oMm8pICU+JSANCiAgbWFwKH5oMm8uYXV0b21sKHkgPSAiYW55X3B1cmNoYXNlIiwgdHJhaW5pbmdfZnJhbWUgPSAuLCBzZWVkID0gMTMxMywgbWF4X3J1bnRpbWVfc2VjcyA9IDE4MCkpIC0+DQogIGJhc2ljX2F1dG9tbA0KDQpzYXZlUkRTKGJhc2ljX2F1dG9tbCwgIi4uL291dHB1dHMvYXV0b21sLnJkcyIpDQpgYGANCg0KYGBge3IgZXZhbD0hZG9fZ2xtbmV0cywgZWNobz1GQUxTRX0NCmJhc2ljX2F1dG9tbDwtcmVhZF9yZHMoIi4uL291dHB1dHMvYXV0b21sLnJkcyIpDQpgYGANCg0KYGBge3J9DQpiYXNpY19hdXRvbWwgJT4lIA0KICBtYXBfZGYofmFzLmRhdGEuZnJhbWUoLkBsZWFkZXJib2FyZCksIC5pZD0ic2V0IikgIC0+DQogIGF1dG9tbF9oaWdobGV2ZWxfcmVzdWx0cw0KYGBgDQoNCiMgRXZhbHVhdGlvbg0K